#include "PageLibPreprocessor.hh"
#include<unistd.h>



PageLibPreprocessor::PageLibPreprocessor()
{}

void PageLibPreprocessor::doProcess(DirScanner &dirScanner)//预处理
{
    readInfoFromFile(dirScanner);
    change();
    buildInvertIndexTable();
    storeOnDisk();

}

void PageLibPreprocessor::readInfoFromFile(DirScanner &dirScanner)//根据配置信息读取网页库和位置偏移库的内容
{
    //int size=page.getFiles().size();
    //cout<<size<<endl;
    //string target;
    //smatch txtMatch;
    int index=1;//真实序号
    //string ss;
    //WebPage webpage;

    int id = 1;//序号
    int j = 0;//控制扫描37个文件
    int offsetBegin = 0;//记录起始位置
    int lenth = 0;//记录长度
    XMLDocument doc;
    string title,link,content;
    WebPage webpage;
    //Simhasher simhasher("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
    while(j<dirScanner.files().size())
    {
    //cout<<_dirScanner.files().size()<<endl;
	doc.LoadFile(dirScanner.files()[j].c_str());
    doc.ErrorID();

    XMLElement* channelElement = doc.FirstChildElement("rss")->FirstChildElement("channel");

    XMLElement* itemElement = channelElement->FirstChildElement("item");
  
    XMLElement* titleElement;
    XMLElement* linkElement;
    XMLElement* contentElement;

        while (itemElement != nullptr) {
            
            titleElement = itemElement->FirstChildElement("title");
            if (titleElement) {
                title = titleElement->GetText();
                webpage._docTitle=title;
            }

            linkElement = itemElement->FirstChildElement("link");
            if (linkElement) {
                link = linkElement->GetText();
                webpage._docUrl=link;
            }

            {
#if 1
            contentElement = itemElement->FirstChildElement("content");
            if (contentElement) {
                content = contentElement->GetText();
                std::regex tagRegex("<[^>]*>");
                content = std::regex_replace(content , tagRegex, "");
                webpage._docContent=content;
            }
            else{
#endif
                contentElement = itemElement->FirstChildElement("description");
                if (contentElement) {
                    content = contentElement->GetText();
                    std::regex tagRegex("<[^>]*>");
                    content = std::regex_replace(content , tagRegex, "");
                    webpage._docContent=content;
                }
               }
            }
            webpage._docId=id;
            _var.push_back(webpage);

#if 0
            
            uint64_t u64 = 0;
            simhasher.make(content, 10, u64);
            uint64_t number=u64;//cutRedundantPages(content);
            //cout<<"111   ";
           if(number != 0 && _judge.count(number) == 0)
            {
            //cout<<"111"<<endl;
            _judge.insert(make_pair(number,true));
            webpage._docId=index;
            //cout<<"111"<<endl;
            _pageLib.push_back(webpage);
            //cout<<webpage._docUrl<<endl;
            index++;
            //cout<<index<<endl;
        }
            cout<<"222"<<endl;

#endif
#if 0
            stringstream item;
            item<< "<doc><docid>" << webpage._docId << "</docid><title>" << webpage._docTitle
               << "</title><link>" << webpage._docUrl << "</link><content>"
               << webpage._docContent<< "</content></doc>";
               //cout<<i+1<<"    "<<content<<endl;
            string txt = item.str();
            //outFile << txt<< std::endl;
            //_files.push_back(txt);
#endif
#if 0
            //创建网页偏移库

            {
                lenth=strlen(txt.c_str());
                auto mypair=make_pair(offsetBegin,lenth);
                _offsetLib.insert(make_pair(index-1,mypair));
                offsetBegin=lenth+offsetBegin+1;
                #if 0
                //cout<<lenth<<endl;
                //cout<<offsetBegin<<endl;
                //if()
                cout<<i<<"    "<<_offsetLib.find(i)->second.first<<"    "<<_offsetLib.find(i)->second.second<<endl;
                //cout<<_offsetLib.find(i)->second<<endl;
                #endif
            }
#endif
            #if 0 
            cout<<"<doc><docid>" << i<< "</docid><title>" << title
               << "</title><link>" << link << "</link><content>"
               << content<< "</content></doc>"<<endl;

            #endif
            //_files.push_back(txt);
           
            itemElement= itemElement->NextSiblingElement("item");
            id++;
        }
        j++;
    }
}



    #if 0
    for(int tem=0;tem<size;tem++)
    {
        
        string item=page.getFiles()[tem];
        cout<<"ll"<<endl;   
        regex titleRegex("<title>(.*?)</title>");
        if (std::regex_search(item, txtMatch, titleRegex)) {
            target = txtMatch.str(1);
            webpage._docTitle=target;
    }
        regex linkRegex("<link>(.*?)</link>");
        if (std::regex_search(item, txtMatch, linkRegex)) {
            target = txtMatch.str(1);
            webpage._docUrl=target;
    }
        regex idRegex("<docid>(.*?)</docid>");
        if (std::regex_search(item, txtMatch, idRegex)) {
            target = txtMatch.str(1);
            int number=stoi(target);
            webpage._docId=number;
    }  

        regex conRegex("<content>(.*?)</content>");
        if (std::regex_search(item, txtMatch, conRegex)) {
            target = txtMatch.str(1);
            webpage._docContent=target;
    }

        uint64_t number=cutRedundantPages(target);
        if(number != 0 && _judge.count(number) ==0)
        {
            //cout<<"111"<<endl;
            _judge.insert(make_pair(number,true));
            webpage._docId=index;
            //cout<<"111"<<endl;
            _pageLib.push_back(webpage);
            //cout<<webpage._docUrl<<endl;
            index++;
            //cout<<index<<endl;
        }
//_pageLib.push_back(webpage);
#if 0
        //cout<<"ok1"<<endl;
        if(_judge.find(cutRedundantPages(webpage))->first != 0 && _judge.find(cutRedundantPages(webpage))->second != true)
        {
            _judge.insert(make_pair(_judge.find(cutRedundantPages(webpage))->second,true));
            webpage._docId=index;
            _pageLib.push_back(webpage);
            //cout<<webpage._docUrl<<endl;
            index++;
            //cout<<index<<endl;
        }
        //cout<<"ok2"<<endl;
#endif
    }
#endif  
    




void PageLibPreprocessor::change()
{
    WebPage webpage;
    int index=1;                       
    stringstream item;
    string txt;
    int offsetBegin = 0;//记录起始位置
    int lenth = 0;//记录长度
    //cout<<_var.size()<<endl;4365
    //for(int i=0;i<1000;i++)
    Simhasher simhasher("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
    for(int i=0;i<_var.size();i++)
    {
        webpage=_var[i];
        //cout<<webpage._docId<<endl;
        //uint64_t number=cutRedundantPages(_var[i]._docContent);
        //if(number != 0 )
        uint64_t u64 = 0;
        simhasher.make(webpage._docContent, 10, u64);
        uint64_t number=u64;//cutRedundantPages(content);
            //cout<<"111   ";
        if(number != 0 && _judge.count(number) == 0)
        {
            //cout<<"111"<<endl;
            _judge.insert(make_pair(number,true));
            webpage._docId=index;
            //cout<<"111"<<endl;
            _pageLib.push_back(webpage);
            //cout<<webpage._docUrl<<endl;
            txt= "<doc><docid>"+to_string(webpage._docId)+"</docid><title>"+webpage._docTitle+"</title><link>"+webpage._docUrl+"</link><content>"+webpage._docContent+"</content></doc>";
            //txt = item.str();
            lenth=strlen(txt.c_str());
            pair<int,int> mypair=make_pair(offsetBegin,lenth);
            //cout<<mypair.first<<" "<<mypair.second<<endl;
            _offsetLib.insert({index,mypair});
            offsetBegin=lenth+offsetBegin;
            index++;
            //cout<<index<<endl;
        }
    #if 0
        if(webpage._docContent.size()!=0)
        {
            //cout<<_judge.count(number)<<endl;
            webpage._docId=index;
            _pageLib.push_back(webpage);
            txt= "<doc><docid>"+to_string(webpage._docId)+"</docid><title>"+webpage._docTitle+"</title><link>"+webpage._docUrl+"</link><content>"+webpage._docContent+"</content></doc>";
            //txt = item.str();
            lenth=strlen(txt.c_str());
            pair<int,int> mypair=make_pair(offsetBegin,lenth);
            //cout<<mypair.first<<" "<<mypair.second<<endl;
            _offsetLib.insert({index,mypair});
            //cout<<_offsetLib.find(i)->first<<" "
            //    <<_offsetLib.find(i)->second.first<<" "
             //   <<_offsetLib.find(i)->second.second<<endl;
            //cout<<index<<" "
            //    <<offsetBegin<<" "
            //    <<lenth<<endl;
            offsetBegin=lenth+offsetBegin+1;
            index++;
        }
    #endif
    }


}

uint64_t PageLibPreprocessor::cutRedundantPages(string &item)//对冗余的网页进行去重
{
    //cout<<"time1    ";
    Simhasher simhasher("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
    uint64_t u64 = 0;
    simhasher.make(item, 1, u64);
    //cout<<"time2"<<endl;
    return u64;
}




void PageLibPreprocessor::buildInvertIndexTable()//创建倒排索引表
{   
    InvertedIndex invertedIndex("  "," ","../data/invertIndex.lib","../conf/stop_words_zh.txt","../conf/stop_words_eng.txt"
                                ,_offsetLib,_pageLib);
    invertedIndex.buildInvertedIndexTable();
}


void PageLibPreprocessor::storeOnDisk()//将经过预处理之后的网页库、位置偏移库和倒排索引表写回到磁盘上
{
    //WebPage webpage;
    string ss;
    std::ofstream outFile("../data/tipepage.lib"); // 创建并打开文件
    std::ofstream offSet("../data/offset.lib");
    //std::ofstream index("../data/invertIndex.lib");
            for(int i=1;i<=_pageLib.size();i++)
            {
            //webpage=_pageLib[i];
            
            outFile<< "<doc><docid>" << _pageLib[i-1]._docId << "</docid><title>" << _pageLib[i-1]._docTitle
                << "</title><link>" << _pageLib[i-1]._docUrl<< "</link><content>"
                << _pageLib[i-1]._docContent<< "</content></doc>";
            offSet<<_offsetLib.find(i)->first<<" "
                  <<_offsetLib.find(i)->second.first<<" "
                  <<_offsetLib.find(i)->second.second<<endl;
            }

    outFile.close();
    offSet.close();
}


#if 0
int main()
{
    DirScanner _dirScanner;
    _dirScanner();
    PageLib page(_dirScanner);
    page.create();
    PageLibPreprocessor process;
    process.doProcess(page);


    return 0;


}

#endif
